{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d00fc083",
   "metadata": {
    "papermill": {
     "duration": 0.002622,
     "end_time": "2022-10-26T08:27:02.102770",
     "exception": false,
     "start_time": "2022-10-26T08:27:02.100148",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# input-url"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "645dc2c4",
   "metadata": {
    "papermill": {
     "duration": 0.004143,
     "end_time": "2022-10-26T08:27:02.111011",
     "exception": false,
     "start_time": "2022-10-26T08:27:02.106868",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This component reads a file from a HTTP(s) source via wget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e59fe38",
   "metadata": {
    "papermill": {
     "duration": 2.308259,
     "end_time": "2022-10-26T08:27:04.423791",
     "exception": false,
     "start_time": "2022-10-26T08:27:02.115532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install wget==3.2 requests==2.28.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "054a358d",
   "metadata": {
    "papermill": {
     "duration": 0.013833,
     "end_time": "2022-10-26T08:27:04.440049",
     "exception": false,
     "start_time": "2022-10-26T08:27:04.426216",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import re\n",
    "import requests\n",
    "import sys\n",
    "import wget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcab9b87",
   "metadata": {
    "papermill": {
     "duration": 0.013675,
     "end_time": "2022-10-26T08:27:04.455836",
     "exception": false,
     "start_time": "2022-10-26T08:27:04.442161",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# path and file name for output\n",
    "output_data = os.environ.get('output_data', 'data.csv')\n",
    "\n",
    "# url of souce\n",
    "url = os.environ.get('url')\n",
    "\n",
    "# temporal data storage for local execution\n",
    "data_dir = os.environ.get('data_dir', '../../data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22c51c11-7c47-4877-92a3-88e071506012",
   "metadata": {
    "papermill": {
     "duration": 0.01535,
     "end_time": "2022-10-26T08:27:04.474938",
     "exception": false,
     "start_time": "2022-10-26T08:27:04.459588",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "parameters = list(\n",
    "    map(lambda s: re.sub('$', '\"', s),\n",
    "        map(\n",
    "            lambda s: s.replace('=', '=\"'),\n",
    "            filter(\n",
    "                lambda s: s.find('=') > -1 and bool(re.match(r'[A-Za-z0-9_]*=[.\\/A-Za-z0-9]*', s)),\n",
    "                sys.argv\n",
    "            )\n",
    "    )))\n",
    "\n",
    "for parameter in parameters:\n",
    "    logging.debug('Parameter: ' + parameter)\n",
    "    exec(parameter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "307d42c7-d81b-4bf1-b823-431e7c434631",
   "metadata": {
    "papermill": {
     "duration": 0.965251,
     "end_time": "2022-10-26T08:27:05.463197",
     "exception": false,
     "start_time": "2022-10-26T08:27:04.497946",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "header={\"user-agent\":\"Mozilla/5.0 (Windows NT 6.0) Gecko/20100101 Firefox/14.0.1\"}\n",
    "r = requests.get(url, headers=header)\n",
    "print(output_data)\n",
    "\"\"\"\n",
    "logging.debug('output_data: ' + output_data)\n",
    "os.remove(os.path.join(output_data)) if os.path.exists(os.path.join(output_data)) else None\n",
    "head, _ = os.path.split(output_data)\n",
    "os.makedirs(head)\n",
    "\"\"\"\n",
    "with open('/data/result', \"wb\") as f:\n",
    "    f.write(r.content)\n",
    "with open(output_data, \"w\") as f:\n",
    "    f.write('/data/result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "665247fb",
   "metadata": {
    "papermill": {
     "duration": 0.012156,
     "end_time": "2022-10-26T08:27:05.478049",
     "exception": false,
     "start_time": "2022-10-26T08:27:05.465893",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "print('Data written successfully')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 5.223558,
   "end_time": "2022-10-26T08:27:05.697642",
   "environment_variables": {},
   "exception": null,
   "input_path": "/home/romeokienzler/gitco/claimed/component-library/input/input-url.ipynb",
   "output_path": "/home/romeokienzler/gitco/claimed/component-library/input/input-url.ipynb",
   "parameters": {},
   "start_time": "2022-10-26T08:27:00.474084",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
